package com.lsc.jiujia.util;

import com.lsc.jiujia.entity.IPEntity;
import com.lsc.jiujia.service.IPPool;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHost;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import com.lsc.jiujia.service.IPThread;
import org.springframework.stereotype.Service;

@Slf4j
@Service
public class IPUtil {

    public static void IPIsable(List<IPEntity> ipEntityList) {
        CloseableHttpClient httpClient = HttpClients.createDefault();
        CloseableHttpResponse response = null;

        for(int i = 0; i < ipEntityList.size(); i++) {
            String ip = ipEntityList.get(i).getIPAddress();
            String port = ipEntityList.get(i).getIPPort();

            HttpHost proxy = new HttpHost(ip, Integer.parseInt(port));
            RequestConfig config = RequestConfig.custom().setProxy(proxy).setConnectTimeout(5000).
                    setSocketTimeout(5000).build();
            HttpGet httpGet = new HttpGet("https://www.baidu.com");
            httpGet.setConfig(config);

            httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;" +
                    "q=0.9,image/webp,*/*;q=0.8");
            httpGet.setHeader("Accept-Encoding", "gzip, deflate, sdch");
            httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.8");
            httpGet.setHeader("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit" +
                    "/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

            try {
                response = httpClient.execute(httpGet);
            } catch (IOException e) {
                log.info("不可用代理已删除" + ipEntityList.get(i).getIPAddress()
                        + ": " + ipEntityList.get(i).getIPPort());
                ipEntityList.remove(ipEntityList.get(i));
                i--;
            }
        }

        try {
            if (httpClient != null) {
                httpClient.close();
            }
            if (response != null) {
                response.close();
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }


    /*//对IP进行过滤
    public static List<IPEntity> Filter(List<IPEntity> ipEntityList) {
        List<IPEntity> newIPMessages = new ArrayList<>();

        for (int i = 0; i < ipEntityList.size(); i++) {
            String ipType = ipEntityList.get(i).getIPType();
            String ipSpeed = ipEntityList.get(i).getIPSpeed();

            ipSpeed = ipSpeed.substring(0, ipSpeed.indexOf('秒'));
            double Speed = Double.parseDouble(ipSpeed);

            if (ipType.equals("HTTPS") && Speed <= 2.0) {
                newIPMessages.add(ipEntityList.get(i));
            }
        }

        return newIPMessages;
    }*/

    //使用本机IP爬取xici代理网站的第一页
    public static List<IPEntity> urlParse(List<IPEntity> ipEntityList) {
        String url = "http://www.kuaidaili.com/proxylist/1";
        String html = getHtml(url);

        //将html解析成DOM结构
        Document document = Jsoup.parse(html);

        //提取所需要的数据
        Elements trs = document.select("table").select("tbody").select("tr");
        for (int i = 5; i < trs.size(); i++) {
            IPEntity ipEntity = new IPEntity();
            String ipAddress = trs.get(i).select("td").get(0).text();
            String ipPort = trs.get(i).select("td").get(1).text();
            String ipType = trs.get(i).select("td").get(3).text();
            String ipSpeed = trs.get(i).select("td").get(6).text();

            ipEntity.setIPAddress(ipAddress);
            ipEntity.setIPPort(ipPort);
            ipEntity.setIPType(ipType);
            ipEntity.setIPSpeed(ipSpeed);

            ipEntityList.add(ipEntity);
        }

        return ipEntityList;
    }

    public static boolean urlParse(String url, String ip, String port,
                                   List<IPEntity> ipMessages1) {
        //调用一个类使其返回html源码
        String html = getHtml(url, ip, port);

        if(html != null) {
            //将html解析成DOM结构
            Document document = Jsoup.parse(html);

            //提取所需要的数据
            Elements trs = document.select("table").select("tbody").select("tr");

            for (int i = 5; i < trs.size(); i++) {
                IPEntity ipMessage = new IPEntity();
                String ipAddress = trs.get(i).select("td").get(0).text();
                String ipPort = trs.get(i).select("td").get(1).text();
                String ipType = trs.get(i).select("td").get(3).text();
                String ipSpeed = trs.get(i).select("td").get(6).text();

                ipMessage.setIPAddress(ipAddress);
                ipMessage.setIPPort(ipPort);
                ipMessage.setIPType(ipType);
                ipMessage.setIPSpeed(ipSpeed);


                ipMessages1.add(ipMessage);
            }

            return true;
        } else {
            log.info(ip+ ": " + port + " 代理不可用");

            return false;
        }
    }


    public static String getHtml( String url, String ip, String port) {
        String entity = null;
        CloseableHttpClient httpClient = HttpClients.createDefault();

        //设置代理访问和超时处理
        log.info("此时线程: " + Thread.currentThread().getName() + " 爬取所使用的代理为: "
                + ip + ":" + port);
        HttpHost proxy = new HttpHost(ip, Integer.parseInt(port));
        RequestConfig config = RequestConfig.custom().setProxy(proxy).setConnectTimeout(3000).
                setSocketTimeout(3000).build();
        HttpGet httpGet = new HttpGet(url);
        httpGet.setConfig(config);

        httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;" +
                "q=0.9,image/webp,*/*;q=0.8");
        httpGet.setHeader("Accept-Encoding", "gzip, deflate, sdch");
        httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.8");
        httpGet.setHeader("Cache-Control", "no-cache");
        httpGet.setHeader("Connection", "keep-alive");
        httpGet.setHeader("Host", "www.kuaidaili.com");
        httpGet.setHeader("Pragma", "no-cache");
        httpGet.setHeader("Upgrade-Insecure-Requests", "1");
        httpGet.setHeader("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " +
                "(KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

        try {
            //客户端执行httpGet方法，返回响应
            CloseableHttpResponse httpResponse = httpClient.execute(httpGet);

            //得到服务响应状态码
            if (httpResponse.getStatusLine().getStatusCode() == 200) {
                entity = EntityUtils.toString(httpResponse.getEntity(), "utf-8");
            }

            httpResponse.close();
            httpClient.close();
        } catch (ClientProtocolException e) {
            entity = null;
        } catch (IOException e) {
            entity = null;
        }

        return entity;
    }

    //对上一个方法的重载，使用本机ip进行网站爬取
    public static String getHtml(String url) {
        String entity = null;
        CloseableHttpClient httpClient = HttpClients.createDefault();

        //设置超时处理
        RequestConfig config = RequestConfig.custom().setConnectTimeout(3000).
                setSocketTimeout(3000).build();
        HttpGet httpGet = new HttpGet(url);
        httpGet.setConfig(config);

        httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;" +
                "q=0.9,image/webp,*/*;q=0.8");
        httpGet.setHeader("Accept-Encoding", "gzip, deflate, sdch");
        httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.8");
        httpGet.setHeader("Cache-Control", "no-cache");
        httpGet.setHeader("Connection", "keep-alive");
        httpGet.setHeader("Host", "www.kuaidaili.com");
        httpGet.setHeader("Pragma", "no-cache");
        httpGet.setHeader("Upgrade-Insecure-Requests", "1");
        httpGet.setHeader("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " +
                "(KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

        try {
            //客户端执行httpGet方法，返回响应
            CloseableHttpResponse httpResponse = httpClient.execute(httpGet);

            //得到服务响应状态码
            if (httpResponse.getStatusLine().getStatusCode() == 200) {
                entity = EntityUtils.toString(httpResponse.getEntity(), "utf-8");
            }

            httpResponse.close();
            httpClient.close();
        } catch (ClientProtocolException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }

        return entity;
    }

    /**
     * 获取有效IP集合
     */
    public static List<IPEntity> getIPList(){
        List<IPEntity> ipEntityList = new ArrayList<>();
        List<String> urls = new ArrayList<>();
        //首先使用本机ip爬取代理网第一页
        ipEntityList = urlParse(ipEntityList);
        List<Thread> threads = new ArrayList<>();

        //对得到的IP进行筛选，将IP速度在两秒以内的并且类型是https的留下，其余删除
        //ipEntityList = Filter(ipEntityList);

        //对拿到的ip进行质量检测，将质量不合格的ip在List里进行删除
        IPIsable(ipEntityList);

        //构造种子url(4000条ip)
        for (int i = 2; i <= 20; i++) {
            urls.add("https://www.kuaidaili.com/free/inha/" + i);
        }
        IPPool ipPool = new IPPool(ipEntityList);
        for (int i = 0; i < 8; i++) {
            //给每个线程进行任务的分配
            Thread IPThread = new IPThread(urls.subList(i*2, i*2+2), ipPool);
            threads.add(IPThread);
            IPThread.start();
        }

        /*for (Thread thread : threads) {
            try {
                thread.join();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }*/
        log.info("获取IP结束！");

        return ipEntityList;
    }

}
